package server;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.zip.GZIPInputStream;

import model.Articles;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import com.google.appengine.api.datastore.Text;
import com.googlecode.objectify.Objectify;
import com.googlecode.objectify.ObjectifyService;

import controller.PutCache;

public class GetContent {
	public static void Content()
	{
		String charset="utf-8";
		Document doc = Jsoup.parse(GetHtml("http://goarticles.com/recent-articles/", charset));
		String link=doc.select(".article_title_link").get(0).attr("href");
		
		doc = Jsoup.parse(GetHtml("http://goarticles.com"+link, charset));
		Elements elements = doc.select(".art_head");
		
		String author = elements.select("em").text().replace("by", "").trim();  
		elements.select("em").remove();
		
		String alias = Utils.replace(elements.text());
		String title="";
		try {
			title = Translate(elements.text(), "ja");
		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		
		elements=doc.select(".article");
		elements.select("h1,h3,h2,div").remove();
		elements.select("p").last().remove();
		elements.select("p").last().remove();
		elements.select("a").remove();
		
		String content=elements.html();
		content=content.replaceAll("<p>", "12345|").replaceAll("</p>", "|23456");
		try {
			content = Translate(content, "ja");
			content = content.replaceAll("12345 \\|", "<p>").replaceAll("\\| 23456 ", "</p>");
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		
		try {
			ObjectifyService.register(Articles.class);
		} catch (Exception e) {
			// TODO: handle exception
		}
		
		Articles art = new Articles();
		art.setAlias(alias);
		art.setTitle(new Text(title));
		art.setAuthor(author);
		art.setAuthorAlias(Utils.replace(author));
		art.setContent(new Text(content));

		PutCache.putCacheArticles(art, "recent",10);
		
		Objectify objectify = ObjectifyService.begin();
		objectify.put(art);
		
	}
	
	public static void main(String[] args) {
		Content();
	}
	public static String Translate(String string,String language) throws IOException
	{
		string = URLEncoder.encode(string.replaceAll("[\\']{1}", "\\\'").replaceAll("[\\\"]{1}", "\\\""), "UTF-8");
		URL url = new URL("http://translate.google.com/?hl=en&tab=TT");
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        connection.setDoOutput(true);
        connection.setRequestMethod("POST");
        connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.21) Gecko/20110830 AlexaToolbar/alxf-2.14 AlexaToolbar/pXVUnvuf-1.2 Firefox/3.6.21");
		connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
		connection.setRequestProperty("Accept-Language", "en-us,en;q=0.5");
		connection.setRequestProperty("Accept-Charset", "	ISO-8859-1,utf-8;q=0.7,*;q=0.7");
		connection.setRequestProperty("Connection", "keep-alive");
		connection.setRequestProperty("Keep-Alive", "115");
		connection.setRequestProperty("Cookie", "PREF=ID=987bb7a56b1cc7bc:U=691fa246d56f502e:FF=0:TM=1314876638:LM=1314876681:S=yBXzqp_WQq9ykX-j; NID=50=nfNXg7ymSSsfdzMnkEuo7t1GCgIEwcK5xmpFhu1mQWwvNI5nFHLZpRsy_1XnaoKrdj9hnz7za9-M2OIO99phfFSTmXln-xCvqzJaRWHsN0EQKgpmkvgUDTWPVF4I4c9L; SID=DQAAALcAAADhqr-J8Ls3CKhmbRjwTMMP1DjmIvR8qBFxv99saTaFFgoRsVf18W4p_BTocYoxYNMvsvh9sfjMj7KqLyd_s9Y0lwYY_Rrljd7t5IpKBTWWnsv9VSJ_qelH943zV8YIT3uuIdJbHhFlCnVEu_YealZoldWjf4VirYR0PIrr0i4_JU2JCHRmpyW0ACt3KcyLkukJlGDyT5yC5js3CBSTAhnWSylBpMWPWyWJzqXe2BQFcdfuU_qo00pAR-uEcMQ437M; HSID=A9Z-j3CKhzDEykAES");
		connection.setRequestProperty("Cache-Control", "max-age=0");
		OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream());
        writer.write("text=" + string);
        writer.write("&eotf=1");
        writer.write("&hl=vi");
        writer.write("&ie=UTF-8");
        writer.write("&js=n");
        writer.write("&layout=2");
        writer.write("&prev=_t");
        writer.write("&sl=en");
        writer.write("&tl="+language);
        writer.close();
        if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
        	String content = "";			
        	BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
			while(1==1)
			{
				String str = reader.readLine();
				if(str==null) break;
				content+=str;				
			}
			Document doc = Jsoup.parse(content);
			string = doc.select("#result_box").text().replaceAll("<[ ]*/[ ]+", "</");
        }
        return string;
	}
	public static String GetHtml(String link,String charset){
		String content="";
		
		URL url;
		try {
			url = new URL(link);
			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		    connection.setInstanceFollowRedirects(true);
		    connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.21) Gecko/20110830 AlexaToolbar/alxf-2.14 AlexaToolbar/pXVUnvuf-1.2 Firefox/3.6.21");
			connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			connection.setRequestProperty("Accept-Language", "en-us,en;q=0.5");
			connection.setRequestProperty("Accept-Charset", "	ISO-8859-1,utf-8;q=0.7,*;q=0.7");
			connection.setRequestProperty("Connection", "keep-alive");
			connection.setRequestProperty("Keep-Alive", "115");
			connection.setRequestProperty("Cookie", "PREF=ID=987bb7a56b1cc7bc:U=691fa246d56f502e:FF=0:TM=1314876638:LM=1314876681:S=yBXzqp_WQq9ykX-j; NID=50=nfNXg7ymSSsfdzMnkEuo7t1GCgIEwcK5xmpFhu1mQWwvNI5nFHLZpRsy_1XnaoKrdj9hnz7za9-M2OIO99phfFSTmXln-xCvqzJaRWHsN0EQKgpmkvgUDTWPVF4I4c9L; SID=DQAAALcAAADhqr-J8Ls3CKhmbRjwTMMP1DjmIvR8qBFxv99saTaFFgoRsVf18W4p_BTocYoxYNMvsvh9sfjMj7KqLyd_s9Y0lwYY_Rrljd7t5IpKBTWWnsv9VSJ_qelH943zV8YIT3uuIdJbHhFlCnVEu_YealZoldWjf4VirYR0PIrr0i4_JU2JCHRmpyW0ACt3KcyLkukJlGDyT5yC5js3CBSTAhnWSylBpMWPWyWJzqXe2BQFcdfuU_qo00pAR-uEcMQ437M; HSID=A9Z-j3CKhzDEykAES");
			connection.setRequestProperty("Cache-Control", "max-age=0");
			BufferedReader reader=null;
			if("gzip".equals(connection.getContentEncoding()))
		    {
				reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(connection.getInputStream()), Charset.forName(charset)));
		    } else {
		    	reader= new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName(charset)));
		    }
			char[] buf = new char[102102];
			int numRead=0;
	        while((numRead=reader.read(buf)) != -1){
	            String readData = String.valueOf(buf, 0, numRead);
	            content += readData;
	            buf = new char[102102];
	        }
	        reader.close();
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		return content;
	}
}
